--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Commit 99b3630e7206d82d6f27969ffc9ff020d4bf3439
Parents : 0606afd
Author : Ivan <e46112d44649266d71fe2193e00a4710>
Signature : T66BB85Valid, signed by author
Date : 2026-07-26T05:57:49-05:00
feat: add Python script for fast tree manifest generation and update shell script to utilize it
Changes
4 files changed, 228 insertions(+), 0 deletions(-)
Diff
diff --git a/meshchatx.rsm b/meshchatx.rsm
index f39c9087..185384c2 100644
Binary files a/meshchatx.rsm and b/meshchatx.rsm differ
diff --git a/scripts/ci/tree-manifest.sh b/scripts/ci/tree-manifest.sh
index 45765258..c2131df1 100755
--- a/scripts/ci/tree-manifest.sh
+++ b/scripts/ci/tree-manifest.sh
@@ -63,6 +63,10 @@ tracked_paths() {
}
generate() {
+ if command -v python3 >/dev/null 2>&1 && [ -f "$ROOT/scripts/ci/tree_manifest_generate.py" ]; then
+ python3 "$ROOT/scripts/ci/tree_manifest_generate.py"
+ return $?
+ fi
printf '%s\n' "$MANIFEST_HEADER"
tracked_paths | while IFS= read -r f; do
[ -n "$f" ] || continue
diff --git a/scripts/ci/tree_manifest_generate.py b/scripts/ci/tree_manifest_generate.py
new file mode 100644
index 00000000..1d9b7ea2
--- /dev/null
+++ b/scripts/ci/tree_manifest_generate.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: 0BSD
+
+"""Fast git-index tree manifest for meshchatx.rsm signing.
+
+Hashes index blobs (same bytes as git show :path) via git cat-file --batch.
+Output format matches scripts/ci/tree-manifest.sh generate.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+MANIFEST_HEADER = "# meshchatx tree manifest v1"
+EXCLUDE_RSM = "meshchatx.rsm"
+ALLOWED_MODES = frozenset({"100644", "100755"})
+
+
+def _repo_root() -> Path:
+ return Path(__file__).resolve().parents[2]
+
+
+def is_excluded_path(path: str) -> bool:
+ if path == EXCLUDE_RSM:
+ return True
+ if path == "vendor" or path.startswith("vendor/"):
+ return True
+ if "/vendor/" in path or path.endswith("/vendor"):
+ return True
+ return False
+
+
+def _parse_ls_files_z(raw: bytes) -> list[tuple[str, str, str]]:
+ entries: list[tuple[str, str, str]] = []
+ for chunk in raw.split(b"\0"):
+ if not chunk:
+ continue
+ tab = chunk.find(b"\t")
+ if tab < 0:
+ continue
+ meta = chunk[:tab].decode("ascii", errors="replace")
+ path = chunk[tab + 1 :].decode("utf-8", errors="surrogateescape")
+ parts = meta.split()
+ if len(parts) < 3:
+ continue
+ mode, oid, _stage = parts[0], parts[1], parts[2]
+ entries.append((path, mode, oid))
+ return entries
+
+
+def _read_batch_blob(stdout, header: bytes) -> bytes | None:
+ parts = header.strip().split()
+ if len(parts) == 2 and parts[1] == b"missing":
+ return None
+ if len(parts) != 3:
+ raise RuntimeError(f"unexpected cat-file header: {header!r}")
+ _oid, typ, size_s = parts
+ if typ == b"missing":
+ return None
+ size = int(size_s)
+ data = stdout.read(size)
+ if len(data) != size:
+ raise RuntimeError("short read from git cat-file --batch")
+ if typ == b"blob":
+ delim = stdout.read(1)
+ if delim != b"\n":
+ raise RuntimeError("expected newline delimiter after cat-file blob")
+ return data
+
+
+def generate_manifest(root: Path) -> str:
+ env = os.environ.copy()
+ env["LC_ALL"] = "C"
+ raw = subprocess.check_output(
+ ["git", "ls-files", "-s", "-z"],
+ cwd=root,
+ env=env,
+ )
+ rows: list[tuple[str, str, str]] = []
+ for path, mode, oid in _parse_ls_files_z(raw):
+ if not path or is_excluded_path(path):
+ continue
+ if mode not in ALLOWED_MODES:
+ continue
+ rows.append((path, mode, oid))
+ rows.sort(key=lambda item: item[0])
+
+ lines = [MANIFEST_HEADER]
+ if not rows:
+ return "\n".join(lines) + "\n"
+
+ proc = subprocess.Popen(
+ ["git", "cat-file", "--batch"],
+ cwd=root,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ env=env,
+ )
+ assert proc.stdin is not None
+ assert proc.stdout is not None
+
+ stdin_buf = "".join(f"{oid}\n" for _path, _mode, oid in rows).encode("ascii")
+ proc.stdin.write(stdin_buf)
+ proc.stdin.close()
+
+ for path, _mode, _oid in rows:
+ header = proc.stdout.readline()
+ if not header:
+ raise RuntimeError("git cat-file --batch closed stdout early")
+ blob = _read_batch_blob(proc.stdout, header)
+ if blob is None:
+ continue
+ digest = hashlib.sha256(blob).hexdigest()
+ lines.append(f"{digest} {path}")
+
+ proc.wait()
+ if proc.returncode not in (0, None):
+ raise RuntimeError(f"git cat-file --batch exited {proc.returncode}")
+
+ return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+ root = _repo_root()
+ try:
+ sys.stdout.write(generate_manifest(root))
+ except Exception as exc:
+ print(f"tree_manifest_generate.py: {exc}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/backend/test_tree_manifest_generate.py b/tests/backend/test_tree_manifest_generate.py
new file mode 100644
index 00000000..178e138f
--- /dev/null
+++ b/tests/backend/test_tree_manifest_generate.py
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: 0BSD
+
+import subprocess
+from pathlib import Path
+
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def _slow_manifest() -> str:
+ proc = subprocess.run(
+ ["sh", "scripts/ci/tree-manifest.sh", "generate"],
+ cwd=REPO,
+ capture_output=True,
+ text=True,
+ env={
+ **__import__("os").environ,
+ "PATH": __import__("os").environ.get("PATH", ""),
+ },
+ check=False,
+ )
+ # Force slow path by temporarily hiding the python helper name is awkward.
+ # Compare fast output to a golden re-run via inline shell loop instead.
+ assert proc.returncode == 0, proc.stderr
+ return proc.stdout
+
+
+def _fast_manifest() -> str:
+ proc = subprocess.run(
+ ["python3", "scripts/ci/tree_manifest_generate.py"],
+ cwd=REPO,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return proc.stdout
+
+
+def _legacy_shell_manifest() -> str:
+ """Same rules as tree-manifest.sh generate before the Python fast path."""
+ header = "# meshchatx tree manifest v1"
+ lines = [header]
+ env = {"LC_ALL": "C"}
+ paths = subprocess.check_output(["git", "ls-files"], cwd=REPO, env=env, text=True)
+ for f in sorted(paths.splitlines(), key=lambda s: s):
+ if not f:
+ continue
+ if f == "meshchatx.rsm":
+ continue
+ if f == "vendor" or f.startswith("vendor/"):
+ continue
+ if "/vendor/" in f or f.endswith("/vendor"):
+ continue
+ check = subprocess.run(
+ ["git", "cat-file", "-e", f":{f}"],
+ cwd=REPO,
+ capture_output=True,
+ )
+ if check.returncode != 0:
+ continue
+ mode = subprocess.check_output(
+ ["git", "ls-files", "-s", "--", f],
+ cwd=REPO,
+ text=True,
+ ).split()[0]
+ if mode not in ("100644", "100755"):
+ continue
+ digest = subprocess.check_output(
+ ["sh", "-c", "git show \":$1\" | sha256sum | awk '{print $1}'", "_", f],
+ cwd=REPO,
+ text=True,
+ ).strip()
+ lines.append(f"{digest} {f}")
+ return "\n".join(lines) + "\n"
+
+
+def test_fast_manifest_matches_legacy_shell_algorithm():
+ legacy = _legacy_shell_manifest()
+ fast = _fast_manifest()
+ assert fast == legacy
+
+
+def test_tree_manifest_sh_uses_fast_generator():
+ via_sh = _slow_manifest()
+ fast = _fast_manifest()
+ assert via_sh == fast
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────